Histogram Model (1D) for Image Objects


Sergei Papulin (papulin.study@yandex.ru)

Contents

  • Loading Dataset
    • Image and Annotation
    • Image Objects
  • Defining Positional Elements
    • Basic Elements
    • Position Mask
    • High-Level Elements
  • Defining Object Elements
    • Basic Elements
    • Object Mask
    • High-Level Elements
  • Creating Histogram
  • Querying
    • Set Operations
    • Logical Operations
  • Image Retrieval
  • References

Load packages

Install the following packages if needed: pip install Cython pycocotools scikit-image

In [1]:
import numpy as np
from pycocotools.coco import COCO

import matplotlib.pyplot as plt
from matplotlib.collections import PatchCollection
from matplotlib.patches import Polygon, Rectangle

import skimage.io as io
import skimage.draw as draw

%matplotlib inline
In [2]:
%load_ext autoreload
%autoreload 2

import sys
sys.path.insert(0, "../../")

from lshist.histogram import operations, Histogram1D, HElement
from lshist.executor import Parser, Evaluator
from lshist.utils import E

Loading Dataset

Download the dataset from the COCO website:

Image and Annotation

In [3]:
ANNOT_FILE_PATH = "datasets/annotations/instances_val2017.json"
IMAGE_PATH = "datasets/val2017"
IMAGE_ID = 404484
In [4]:
coco = COCO(ANNOT_FILE_PATH)
loading annotations into memory...
Done (t=0.66s)
creating index...
index created!
In [5]:
img_meta = coco.loadImgs(ids=[IMAGE_ID])[0]
img_meta
Out[5]:
{'coco_url': 'http://images.cocodataset.org/val2017/000000404484.jpg',
 'date_captured': '2013-11-14 17:43:15',
 'file_name': '000000404484.jpg',
 'flickr_url': 'http://farm3.staticflickr.com/2645/4216057008_f835cc6fdf_z.jpg',
 'height': 240,
 'id': 404484,
 'license': 5,
 'width': 320}
In [6]:
I = io.imread("{}/{}".format(IMAGE_PATH, img_meta["file_name"]))
plt.imshow(I)
plt.show()

Image Objects

In [7]:
cats = coco.loadCats(coco.getCatIds())
len_cats = len(cats)
print("Single category:\n{}\n".format(cats[0]))
print("All categories:\n{}\n".format(" ".join([cat["name"] for cat in cats])))
print("Total number of categories: {}".format(len_cats))
Single category:
{'id': 1, 'name': 'person', 'supercategory': 'person'}

All categories:
person bicycle car motorcycle airplane bus train truck boat traffic light fire hydrant stop sign parking meter bench bird cat dog horse sheep cow elephant bear zebra giraffe backpack umbrella handbag tie suitcase frisbee skis snowboard sports ball kite baseball bat baseball glove skateboard surfboard tennis racket bottle wine glass cup fork knife spoon bowl banana apple sandwich orange broccoli carrot hot dog pizza donut cake chair couch potted plant bed dining table toilet tv laptop mouse remote keyboard cell phone microwave oven toaster sink refrigerator book clock vase scissors teddy bear hair drier toothbrush

Total number of categories: 80
In [8]:
img_anns_id = coco.getAnnIds(imgIds=IMAGE_ID, iscrowd=None)
img_anns = coco.loadAnns(img_anns_id)
print(img_anns[0])
{'iscrowd': 0, 'area': 2613.772699999999, 'image_id': 404484, 'segmentation': [[115.46, 164.53, 106.2, 165.3, 105.95, 156.82, 104.15, 155.28, 103.63, 149.37, 104.15, 140.88, 99.78, 139.34, 96.69, 136.0, 94.12, 130.86, 92.84, 123.66, 94.12, 118.52, 94.38, 116.21, 88.47, 105.41, 86.93, 98.99, 90.27, 92.82, 95.92, 90.76, 101.58, 93.08, 108.77, 97.7, 111.6, 104.64, 111.86, 105.93, 117.51, 105.67, 127.54, 105.16, 136.27, 107.73, 145.53, 107.47, 151.7, 109.27, 156.58, 108.5, 163.01, 114.15, 163.52, 116.47, 167.63, 118.52, 169.43, 121.61, 164.81, 123.15, 163.26, 128.03, 161.98, 129.32, 158.89, 132.15, 155.81, 134.97, 149.9, 133.69, 146.56, 132.15, 148.1, 139.6, 151.7, 150.65, 154.52, 154.25, 159.15, 154.76, 157.35, 156.82, 151.95, 158.36, 148.36, 153.74, 140.64, 138.83, 138.59, 136.0, 133.7, 134.97, 132.93, 136.51, 132.93, 141.14, 130.62, 146.54, 129.85, 150.14, 129.85, 152.19, 127.02, 153.99, 125.48, 150.39, 127.28, 146.02, 129.85, 141.4, 127.54, 133.69, 122.91, 131.12, 120.34, 128.03, 118.8, 126.49, 113.4, 127.26, 114.68, 128.03, 113.4, 135.49, 111.86, 139.6, 110.57, 141.91, 109.29, 146.54, 111.09, 150.91, 113.14, 153.48, 115.46, 154.25, 115.46, 156.82, 113.91, 157.08, 110.83, 156.82, 110.32, 153.99, 108.0, 153.74, 109.8, 156.05, 110.32, 161.7, 113.66, 162.99]], 'category_id': 18, 'id': 7981, 'bbox': [86.93, 90.76, 82.5, 74.54]}
In [9]:
seg_polys = list()
poly_colors = list()

for seg in img_anns:
    seg_ = seg["segmentation"][0]
    poly_colors.append((np.random.random((1, 3))*0.5+0.5).tolist()[0])
    seg_polys.append(Polygon(np.array(seg_).reshape((int(len(seg_)/2), 2)), fill=False))
In [10]:
fig, ax = plt.subplots(1) #, figsize=(15,15))

ax.imshow(I)
p_objs = PatchCollection(seg_polys, facecolor=poly_colors, edgecolor=poly_colors, alpha=0.6, linewidths=2)
ax.add_collection(p_objs)
plt.show()

Defining Positional Elements

Basic Elements

In [11]:
def generate_positional_grid_1d(num_x, num_y):
    elements = list()
    for i in range(num_y):
        for j in range(num_x):
            element = dict()
            element["id"] = "e{}".format(i*num_x + j + 1)
            element["pos"] = (j*1/num_x, i*1/num_y, 1/num_x, 1/num_y)
            elements.append(element)
    return elements


def get_positional_grid_1d(width, height, elements):
    elements_abs = list()
    for el in elements:
        x_start = el["pos"][0] * width
        y_start = el["pos"][1] * height
        x_end = x_start + el["pos"][2] * width
        y_end = y_start + el["pos"][3] * height
        elements_abs.append({"id": el["id"], "pos": (x_start, y_start, x_end, y_end)})
    return elements_abs
In [12]:
GRID_X_SPLITS = 5
GRID_Y_SPLITS = 5

grid = generate_positional_grid_1d(GRID_X_SPLITS, GRID_Y_SPLITS)
grid[:2]
Out[12]:
[{'id': 'e1', 'pos': (0.0, 0.0, 0.2, 0.2)},
 {'id': 'e2', 'pos': (0.2, 0.0, 0.2, 0.2)}]
In [13]:
position_elements = get_positional_grid_1d(img_meta["width"], img_meta["height"], grid)
position_elements[:5]
Out[13]:
[{'id': 'e1', 'pos': (0.0, 0.0, 64.0, 48.0)},
 {'id': 'e2', 'pos': (64.0, 0.0, 128.0, 48.0)},
 {'id': 'e3', 'pos': (128.0, 0.0, 192.0, 48.0)},
 {'id': 'e4', 'pos': (192.0, 0.0, 256.0, 48.0)},
 {'id': 'e5', 'pos': (256.0, 0.0, 320.0, 48.0)}]
In [14]:
position_converter = {el["id"]: el["pos"] for el in position_elements}
In [15]:
Up = {el["id"] for el in position_elements}

Show the positional element along with the initial image:

In [16]:
def show_positional_grid(I, elements, position_converter):
    pos_el_rects = list()
    pos_el_texts = list()
    fig, ax = plt.subplots(1) #, figsize=(15,15))
    ax.imshow(I)
    for el in elements:
        pos = position_converter[el]
        left, width = pos[0], pos[2]-pos[0]
        bottom, height = pos[1], pos[3]-pos[1]
        right = left + width
        top = bottom + height
        ax.add_patch(Rectangle(xy=(left, bottom), width=width, height=height, fill=False, 
                                      label=el, edgecolor="red", linewidth=2))
        ax.text(0.5*(left+right), 0.5*(bottom+top), el, 
                horizontalalignment="center", verticalalignment="center", fontsize=15, color="red")
    plt.show()
In [17]:
show_positional_grid(I, Up, position_converter)

Position Mask

In [18]:
def create_position_mask(width, height, position_elements):
    pos_mask = np.zeros((height, width), dtype=np.object) #dtype=np.int)
    # pos_mask = np.chararray((img_meta["height"], img_meta["width"]))
    for pos in position_elements:
        start = [int(pos["pos"][1]), int(pos["pos"][0])]
        end = [int(pos["pos"][3]), int(pos["pos"][2])]
        r, c = draw.rectangle(start, end=end, shape=pos_mask.shape)
        r.dtype = c.dtype = np.int
        pos_mask[r, c] = pos["id"] # int(pos["id"].strip("e"))
    return pos_mask
In [19]:
pos_mask = create_position_mask(img_meta["width"], img_meta["height"], position_elements)
pos_mask
Out[19]:
array([['e1', 'e1', 'e1', ..., 'e5', 'e5', 'e5'],
       ['e1', 'e1', 'e1', ..., 'e5', 'e5', 'e5'],
       ['e1', 'e1', 'e1', ..., 'e5', 'e5', 'e5'],
       ...,
       ['e21', 'e21', 'e21', ..., 'e25', 'e25', 'e25'],
       ['e21', 'e21', 'e21', ..., 'e25', 'e25', 'e25'],
       ['e21', 'e21', 'e21', ..., 'e25', 'e25', 'e25']], dtype=object)
In [20]:
convpos2int = np.vectorize(lambda x: int(x.strip("e")))
plt.imshow(convpos2int(pos_mask))
plt.show()

High-Level Elements

In [21]:
parser = Parser()
In [22]:
Ep_center = E("e7+e8+e9+e12+e13+e14+e17+e18+e19")
Ep_center_set = parser.parse_set(Ep_center.value)
Ep_center_set
Out[22]:
{'e12', 'e13', 'e14', 'e17', 'e18', 'e19', 'e7', 'e8', 'e9'}
In [23]:
# Definition of high-level positional elements

Ep_top = E("e1+e2+e3+e4+e5+e6+e7+e8+e9+e10")
Ep_bottom = E("e16+e17+e18+e19+e20+e21+e22+e23+e24+e25")
Ep_left = E("e1+e2+e6+e7+e11+e12+e16+e17+e21+e22")
Ep_right = E("e4+e5+e9+e10+e14+e15+e19+e20+e24+e25")
Ep_center = E("e7+e8+e9+e12+e13+e14+e17+e18+e19")

Eps = [("top", Ep_top), ("bottom", Ep_bottom), ("left", Ep_left), ("right", Ep_right), ("center", Ep_center)]


# Sets of high-level positional elements (they will be used for the Evaluator below)

Eps_set = { name: parser.parse_set(Ep.value) for name, Ep in Eps}
Eps_set["center"]
Out[23]:
{'e12', 'e13', 'e14', 'e17', 'e18', 'e19', 'e7', 'e8', 'e9'}

Show a grid of the high-level element along with the initial image:

In [24]:
show_positional_grid(I, Eps_set["center"], position_converter)

Show the high-level element based in the position mask:

In [25]:
def show_positional_elements(I, pos_mask, elements):
    mask = np.full((I.shape[0], I.shape[1], 3), fill_value=0, dtype=np.int)
    
    for x in range(I.shape[1]):
        for y in range(I.shape[0]):
            if pos_mask[y,x] in elements:
                mask[y,x] = I[y,x]
    
    fig, ax = plt.subplots(1) #, figsize=(15,15))
    ax.imshow(I)
    ax.imshow(mask, alpha=0.5)
    plt.show()
In [26]:
show_positional_elements(I, pos_mask, Eps_set["center"])

Defining Object Elements

Basic Elements

In [27]:
cats[:2]
Out[27]:
[{'id': 1, 'name': 'person', 'supercategory': 'person'},
 {'id': 2, 'name': 'bicycle', 'supercategory': 'vehicle'}]
In [28]:
Uo = {str(cat["id"]) for cat in cats}

Object Mask for Image

In [29]:
def create_object_mask(width, height, img_anns):
    obj_mask = np.full((height, width), fill_value="null", dtype=np.object) # fill_value=-1, dtype=np.int)
    for i in range(len(img_anns)):
        if img_anns[i]["iscrowd"] == 0:
            seg_ = img_anns[i]["segmentation"][0]
            poly_ = np.array(seg_).reshape((int(len(seg_)/2), 2))
            r, c = draw.polygon(poly_[:,1], poly_[:,0])
            obj_mask[r, c] = str(img_anns[i]["category_id"])
    return obj_mask
In [30]:
obj_mask = create_object_mask(img_meta["width"], img_meta["height"], img_anns)
obj_mask
Out[30]:
array([['null', 'null', 'null', ..., 'null', 'null', 'null'],
       ['null', 'null', 'null', ..., 'null', 'null', 'null'],
       ['null', 'null', 'null', ..., 'null', 'null', 'null'],
       ...,
       ['null', 'null', 'null', ..., 'null', 'null', 'null'],
       ['null', 'null', 'null', ..., 'null', 'null', 'null'],
       ['null', 'null', 'null', ..., 'null', 'null', 'null']],
      dtype=object)
In [31]:
convobj2int = np.vectorize(lambda x: int(x.strip("null") if x.strip("null") else 0))
plt.imshow(convobj2int(obj_mask))
plt.show()

High-Level Elements

In [32]:
catid_by_name = {cat["name"]: cat["id"] for cat in cats}
catid_by_name["person"]
Out[32]:
1
In [33]:
catname_by_id = {cat["id"]: cat["name"] for cat in cats}
catname_by_id[1]
Out[33]:
'person'
In [34]:
Eo_person = E(str(catid_by_name["person"]))
Eo_person_set = parser.parse_set(Eo_person.value)
Eo_person_set
Out[34]:
{'1'}
In [35]:
Eos_set = {cat["name"]: parser.parse_set(E(str(cat["id"])).value) for cat in cats}
Eos_set["person"]
Out[35]:
{'1'}
In [36]:
Eo_pet = E("{}+{}".format(catid_by_name["dog"], catid_by_name["cat"])) 
Eo_pet_set = parser.parse_set(Eo_pet.value)
Eo_pet_set
Out[36]:
{'17', '18'}
In [37]:
Eos_set.update({"pet": Eo_pet_set})
Eos_set["pet"]
Out[37]:
{'17', '18'}

Show a high-level element along with the initial image:

In [38]:
def show_object_segment(I, elements, image_objects):
    seg_polys = list()
    poly_colors = list()
    for seg in image_objects:
        if str(seg["category_id"]) in elements:
            seg_ = seg["segmentation"][0]
            poly_colors.append((np.random.random((1, 3))*0.5+0.5).tolist()[0])
            seg_polys.append(Polygon(np.array(seg_).reshape((int(len(seg_)/2), 2)), fill=False))
    fig, ax = plt.subplots(1) #, figsize=(15,15))
    ax.imshow(I)
    p_objs = PatchCollection(seg_polys, facecolor=poly_colors, edgecolor=poly_colors, alpha=0.6, linewidths=2)
    ax.add_collection(p_objs)
    plt.show()
In [39]:
img_anns_id = coco.getAnnIds(imgIds=IMAGE_ID, iscrowd=None)
img_anns = coco.loadAnns(img_anns_id)
In [40]:
show_object_segment(I, Eos_set["pet"], img_anns)

Show the high-level element based in the object mask:

In [41]:
def show_object_elements(I, obj_mask, elements):
    mask = np.full((I.shape[0], I.shape[1], 3), fill_value=0, dtype=np.int)
    colors = {el: np.random.randint(0, 255, 3) for el in elements}
    for x in range(I.shape[1]):
        for y in range(I.shape[0]):
            if obj_mask[y,x] in elements:
                mask[y,x] = colors[obj_mask[y,x]] #(0,255,156) #colors[obj_mask[y,x]]
    
    fig, ax = plt.subplots(1) #, figsize=(15,15))
    ax.imshow(I)
    ax.imshow(mask, alpha=0.5)
    plt.show()
In [42]:
show_object_elements(I, obj_mask, Eos_set["pet"])

Creating Histogram

In [43]:
def create_histogram(width, height, pos_mask, obj_mask):
    hist = Histogram1D(data=None)
    for x in range(width):
        for y in range(height):
            if obj_mask[y, x] != "null": # if obj_mask[y, x] > 0:
                el_id = (pos_mask[y, x], obj_mask[y, x])
                if el_id not in hist:
                    hist[el_id] = HElement(el_id, 0)
                hist[el_id].value += 1
    hist.normalize(width * height)
    return hist
In [44]:
hist = create_histogram(img_meta["width"], img_meta["height"], pos_mask, obj_mask)
hist.to_dict()
Out[44]:
{('e1', '72'): 9.114583333333334e-05,
 ('e10', '1'): 0.0019921875,
 ('e10', '64'): 0.007005208333333333,
 ('e11', '72'): 0.003203125,
 ('e11', '88'): 0.0012109375,
 ('e12', '18'): 0.016315104166666667,
 ('e12', '88'): 0.0055859375,
 ('e13', '18'): 0.013619791666666667,
 ('e14', '1'): 0.0036067708333333334,
 ('e14', '64'): 0.016848958333333334,
 ('e15', '64'): 0.02421875,
 ('e17', '18'): 0.001953125,
 ('e17', '88'): 0.0003515625,
 ('e18', '18'): 0.0014713541666666666,
 ('e19', '64'): 0.0004427083333333333,
 ('e20', '64'): 0.0014192708333333334,
 ('e3', '1'): 0.0037239583333333335,
 ('e4', '1'): 0.0130078125,
 ('e6', '72'): 0.010208333333333333,
 ('e7', '18'): 0.0006119791666666667,
 ('e8', '1'): 0.0004166666666666667,
 ('e9', '1'): 0.032578125}

Querying

In [45]:
high_level_elements = {
    0: Eps_set, # positions
    1: Eos_set  # objects
}
In [46]:
evaluator = Evaluator(operations, hist, high_level_elements=high_level_elements)
In [47]:
POS1 = "center"
OBJ1 = "person"

POS2 = "left"
OBJ2 = "dog"
In [48]:
E1 = E(POS1, OBJ1)
E2 = E(POS2, OBJ2)
In [49]:
E1_expr = parser.parse_string(E1.value)
HE1 = evaluator.eval(E1_expr)
print("Expression for E1:\n{}".format(E1.value))
print("\nThe parsed expressino for E1 in the postfix notation:\n{}".format(E1_expr))
print("\nHistogram of E1 given the image:\n{}".format(HE1.to_dict()))
print("\nValue of presence for E1:\n{}".format(HE1.sum()))
Expression for E1:
(center,person)

The parsed expressino for E1 in the postfix notation:
[('center', 'person')]

Histogram of E1 given the image:
{('e8', '1'): 0.0004166666666666667, ('e14', '1'): 0.0036067708333333334, ('e9', '1'): 0.032578125}

Value of presence for E1:
0.0366015625
In [50]:
def show_elements(I, pos_mask, obj_mask, pos_elements, obj_elements, title=None):
    mask = np.full((I.shape[0], I.shape[1], 3), fill_value=0, dtype=np.int)
    colors = {el: np.random.randint(0, 255, 3) for el in obj_elements}
    for x in range(I.shape[1]):
        for y in range(I.shape[0]):
            if pos_mask[y,x] in pos_elements:
                mask[y,x] = I[y,x]
                if obj_mask[y,x] in obj_elements:
                    mask[y,x] = colors[obj_mask[y,x]]
    
    fig, ax = plt.subplots(1)
    if title:
        fig.suptitle(title)
    ax.imshow(I)
    ax.imshow(mask, alpha=0.5)
    plt.show()

    
def show_elements_by_HE(I, pos_mask, obj_mask, HE, title=None):
    mask = np.full((I.shape[0], I.shape[1], 3), fill_value=0, dtype=np.int)
    elements = HE.to_dict().keys()
    pos_elements = {el[0] for el in elements}
    obj_elements = {el[1] for el in elements}
    colors = {el: np.random.randint(0, 255, 3) for el in obj_elements}
        
    for x in range(I.shape[1]):
        for y in range(I.shape[0]):
            if pos_mask[y,x] in pos_elements:
                mask[y,x] = I[y,x]
            if (pos_mask[y,x], obj_mask[y,x]) in elements:
                mask[y,x] = colors[obj_mask[y,x]]
    
    fig, ax = plt.subplots(1)
    if title:
        fig.suptitle(title)
    ax.imshow(I)
    ax.imshow(mask, alpha=0.5)
    plt.show()
In [51]:
show_elements(I, pos_mask, obj_mask, Eps_set[POS1], Eos_set[OBJ1], title="E1")
In [52]:
show_elements_by_HE(I, pos_mask, obj_mask, HE1, title="E1")
In [53]:
E2_expr = parser.parse_string(E2.value)
HE2 = evaluator.eval(E2_expr)
print("Expression for E2:\n{}".format(E2.value))
print("\nThe parsed expressino for E2 in the postfix notation:\n{}".format(E2_expr))
print("\nHistogram of E2 given the image:\n{}".format(HE2.to_dict()))
print("\nValue of presence for E2:\n{}".format(HE2.sum()))
Expression for E2:
(left,dog)

The parsed expressino for E2 in the postfix notation:
[('left', 'dog')]

Histogram of E2 given the image:
{('e12', '18'): 0.016315104166666667, ('e7', '18'): 0.0006119791666666667, ('e17', '18'): 0.001953125}

Value of presence for E2:
0.018880208333333332
In [54]:
show_elements(I, pos_mask, obj_mask, Eps_set[POS2], Eos_set[OBJ2], title="E2")
In [55]:
show_elements_by_HE(I, pos_mask, obj_mask, HE2, title="E2")

Set Operations

UNION

In [56]:
E_union = E1 + E2
E_union_expr = parser.parse_string(E_union.value)
HE_union = evaluator.eval(E_union_expr)

print("Expression for E_union:\n{}".format(E_union))
print("\nThe parsed expression for E_union in the postfix notation:\n{}".format(E_union_expr))
print("\nHistogram of E_union given the image:\n{}".format(HE_union.to_dict()))
print("\nValue of presence for E_union:\n{}".format(HE_union.sum()))
Expression for E_union:
((center,person)+(left,dog))

The parsed expression for E_union in the postfix notation:
[('center', 'person'), ('left', 'dog'), '+']

Histogram of E_union given the image:
{('e8', '1'): 0.0004166666666666667, ('e12', '18'): 0.016315104166666667, ('e17', '18'): 0.001953125, ('e7', '18'): 0.0006119791666666667, ('e14', '1'): 0.0036067708333333334, ('e9', '1'): 0.032578125}

Value of presence for E_union:
0.05548177083333333
In [57]:
show_elements_by_HE(I, pos_mask, obj_mask, HE_union, title="E_union")

INTERSECTION

In [58]:
E_intersect = E1 * E2  # or E1.Intersection(E2)
E_intersect_expr = parser.parse_string(E_intersect.value)
HE_intersect = evaluator.eval(E_intersect_expr)

print("Expression for E_intersect:\n{}".format(E_intersect))
print("\nThe parsed expression for E_intersect in the postfix notation:\n{}".format(E_intersect_expr))
print("\nHistogram of E_intersect given the image:\n{}".format(HE_intersect.to_dict()))
print("\nValue of presence for E_intersect:\n{}".format(HE_intersect.sum()))
Expression for E_intersect:
((center,person)*(left,dog))

The parsed expression for E_intersect in the postfix notation:
[('center', 'person'), ('left', 'dog'), '*']

Histogram of E_intersect given the image:
{}

Value of presence for E_intersect:
0
In [59]:
show_elements_by_HE(I, pos_mask, obj_mask, HE_intersect, title="E_intersect")

SUBSTRACTION or EXCEPTION

In [60]:
E_sub = E1 - E2  # or E1.Sub(E2)
E_sub_expr = parser.parse_string(E_sub.value)
HE_sub = evaluator.eval(E_sub_expr)

print("Expression for E_sub:\n{}".format(E_sub))
print("\nThe parsed expression for E_sub in the postfix notation:\n{}".format(E_sub_expr))
print("\nHistogram of E_sub given the image:\n{}".format(HE_sub.to_dict()))
print("\nValue of presence for E_sub:\n{}".format(HE_sub.sum()))
Expression for E_sub:
((center,person)/(left,dog))

The parsed expression for E_sub in the postfix notation:
[('center', 'person'), ('left', 'dog'), '/']

Histogram of E_sub given the image:
{('e8', '1'): 0.0004166666666666667, ('e14', '1'): 0.0036067708333333334, ('e9', '1'): 0.032578125}

Value of presence for E_sub:
0.0366015625
In [61]:
show_elements_by_HE(I, pos_mask, obj_mask, HE_sub, title="E_sub")

Logical Operations

AND

In [62]:
E_and = E1 & E2  # or E1.And(E2)
E_and_expr = parser.parse_string(E_and.value)
HE_and = evaluator.eval(E_and_expr)

print("Expression for E_and:\n{}".format(E_and))
print("\nThe parsed expression for E_and in the postfix notation:\n{}".format(E_and_expr))
print("\nHistogram of E_and given the image:\n{}".format(HE_and.to_dict()))
print("\nValue of presence for E_and:\n{}".format(HE_and.sum()))
Expression for E_and:
((center,person)&(left,dog))

The parsed expression for E_and in the postfix notation:
[('center', 'person'), ('left', 'dog'), '&']

Histogram of E_and given the image:
{('e12', '18'): 0.016315104166666667, ('e7', '18'): 0.0006119791666666667, ('e17', '18'): 0.001953125}

Value of presence for E_and:
0.018880208333333332
In [63]:
show_elements_by_HE(I, pos_mask, obj_mask, HE_and, title="E_and")

OR

In [64]:
E_or = E1 | E2  # or E1.Or(E2)
E_or_expr = parser.parse_string(E_or.value)
HE_or = evaluator.eval(E_or_expr)

print("Expression for E_or:\n{}".format(E_or))
print("\nThe parsed expression for E_or in the postfix notation:\n{}".format(E_or_expr))
print("\nHistogram of E_or given the image:\n{}".format(HE_or.to_dict()))
print("\nValue of presence for E_or:\n{}".format(HE_or.sum()))
Expression for E_or:
((center,person)|(left,dog))

The parsed expression for E_or in the postfix notation:
[('center', 'person'), ('left', 'dog'), '|']

Histogram of E_or given the image:
{('e8', '1'): 0.0004166666666666667, ('e12', '18'): 0.016315104166666667, ('e17', '18'): 0.001953125, ('e7', '18'): 0.0006119791666666667, ('e14', '1'): 0.0036067708333333334, ('e9', '1'): 0.032578125}

Value of presence for E_or:
0.05548177083333333
In [65]:
show_elements_by_HE(I, pos_mask, obj_mask, HE_or, title="E_or")

XOR

In [66]:
E_xor = E1 ^ E2  # or E1.Xor(E2)
E_xor_expr = parser.parse_string(E_xor.value)
HE_xor = evaluator.eval(E_xor_expr)

print("Expression for E_xor:\n{}".format(E_xor))
print("\nThe parsed expression for E_xor in the postfix notation:\n{}".format(E_xor_expr))
print("\nHistogram of E_xor given the image:\n{}".format(HE_xor.to_dict()))
print("\nValue of presence for E_xor:\n{}".format(HE_xor.sum()))
Expression for E_xor:
((center,person)#|(left,dog))

The parsed expression for E_xor in the postfix notation:
[('center', 'person'), ('left', 'dog'), '#|']

Histogram of E_xor given the image:
{('e8', '1'): 0.0004166666666666667, ('e14', '1'): 0.0036067708333333334, ('e9', '1'): 0.032578125}

Value of presence for E_xor:
0.0366015625
In [67]:
show_elements_by_HE(I, pos_mask, obj_mask, HE_xor, title="E_xor")

XSUBSTRACTION

In [68]:
# TODO

Image Retrieval

Serialize the historgram objects:

In [69]:
# import time
# from IPython.display import clear_output


# LIMIT = 5000

# start_tick = time.time()
# hists = list()

# for indx, (img_id, img_meta) in enumerate(coco.imgs.items()):
#     if indx == LIMIT:
#         break
#     img_anns = coco.imgToAnns[img_id]
#     position_elements = get_positional_grid_1d(img_meta["width"], img_meta["height"], grid)
#     pos_mask = create_position_mask(img_meta["width"], img_meta["height"], position_elements)
#     obj_mask = create_object_mask(img_meta["width"], img_meta["height"], img_anns)
#     hist = create_histogram(img_meta["width"], img_meta["height"], pos_mask, obj_mask)
#     hists.append((img_id, hist))
#     clear_output(wait=True)
#     print("Current image index: {}".format(indx))
# #     if indx % 100 == 0:
# #         print("Current image index: {}".format(indx))

# delta_tick = time.time() - start_tick
# print("Total time: {}s".format(delta_tick))
# print("Time per image: {}s".format(delta_tick / LIMIT))

# with open("imagehist.pickle", "wb") as f:
#     import pickle
#     pickle.dump(hists, f, pickle.HIGHEST_PROTOCOL)

Deserialize the histogram of images:

In [70]:
with open("imagehist.pickle", "rb") as f:
    import pickle
    hists = pickle.load(f)

Define your query:

In [71]:
query = E("left", "dog") & E("center", "person")

Retrieve images using the query:

In [72]:
def retrieve(query, hists, topN=10, lastN=None, threshold=0.001):
    expr = parser.parse_string(query.value)
    HEs = [(img_id, evaluator.eval(expr, hist)) for img_id, hist in hists] 
    img_rank = sorted([(img_id, HE.sum()) for img_id, HE in HEs if HE.sum() > threshold], key=lambda x: -x[1])
    if isinstance(lastN, int):
        return img_rank[:topN], img_rank[-lastN:]
    return img_rank[:topN]


IMAGE_LIMIT = 11
IMAGE_CLMNS = 5


def show_retrieved_images(img_rank, img_paths, limit=None):
    
    img_limit = len(img_rank) if IMAGE_LIMIT > len(img_rank) else IMAGE_LIMIT
    if limit:
        img_limit = limit
    row_num = -(-img_limit // IMAGE_CLMNS)

    fig, axs = plt.subplots(row_num, IMAGE_CLMNS, figsize=(15, 4*row_num), squeeze=False)

    for i in range(row_num):
        for j in range(IMAGE_CLMNS):
            indx = i*IMAGE_CLMNS + j
            if indx >= img_limit:
                fig.delaxes(axs[i,j])
            else:
                I = io.imread(img_paths[indx])
                axs[i,j].imshow(I)
                axs[i,j].set_title("rank={}\nid={}\nscore={:0.4f}".format(indx+1, 
                                                                          img_rank[indx][0], 
                                                                          img_rank[indx][1]))
    plt.tight_layout()
    plt.show()
In [73]:
TOP_N = 20
In [74]:
img_rank = retrieve(query, hists, topN=TOP_N)
img_rank
Out[74]:
[(432553, 0.10032933255269322),
 (236166, 0.09953418427230046),
 (107226, 0.08212565104166666),
 (246454, 0.07708740234374999),
 (117525, 0.072228),
 (479155, 0.06296385542168675),
 (494869, 0.061998682669789223),
 (424162, 0.0439483642578125),
 (371749, 0.041536000000000003),
 (149568, 0.037477231329690344),
 (419974, 0.03142605633802817),
 (375278, 0.02494294294294294),
 (404484, 0.018880208333333332),
 (512836, 0.0187109375),
 (549220, 0.013619791666666667),
 (251572, 0.012870451877934272),
 (579321, 0.012747866465863454),
 (295478, 0.0126220703125),
 (309484, 0.00996423545966229),
 (372819, 0.009862822769953051)]

Show the retrieved images:

In [75]:
img_paths = ["{}/{}".format(IMAGE_PATH, coco.imgs[img_meta_[0]]["file_name"]) for img_meta_ in img_rank]
img_paths[:1]
Out[75]:
['datasets/val2017/000000432553.jpg']
In [76]:
show_retrieved_images(img_rank, img_paths, limit=TOP_N)